package cc.spider.demo.web;

import cc.spider.core.SpiderCore;
import cc.spider.web.WebSpiderExtConfig;
import cc.spider.web.config.Config;
import cc.spider.web.config.Task;
import cc.spider.web.model.GlobalDefaultDataExt;
import cc.spider.web.model.http.Header;
import cc.spider.web.queue.RedisUniqueQueue;

public class WebExtSlave {
    public static void main(String[] args) {
        WebSpiderExtConfig webSpiderExtConfig = new WebSpiderExtConfig("config.json");
        webSpiderExtConfig.setTaskQueue(new RedisUniqueQueue("ad"));
        SpiderCore spiderCore = new SpiderCore(webSpiderExtConfig);
        GlobalDefaultDataExt globalDefaultData = new GlobalDefaultDataExt();
        globalDefaultData.addHeaders(new Header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.106 Safari/537.36"));
        webSpiderExtConfig.setGlobalDefaultData(globalDefaultData);
        Task mainTask = Config.newTask(WebSpiderExtConfig.config.getMainTask());
        spiderCore.addTask(mainTask);
        spiderCore.start();
    }
}
